{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow MNIST Model\n",
    "\n",
    " * Wrap a Tensorflow MNIST python model for use as a prediction microservice in seldon-core\n",
    " \n",
    "   * Run locally on Docker to test\n",
    "   * Deploy on seldon-core running on minikube\n",
    " \n",
    "## Dependencies\n",
    "\n",
    " * [Helm](https://github.com/kubernetes/helm)\n",
    " * [Minikube](https://github.com/kubernetes/minikube)\n",
    " * [S2I](https://github.com/openshift/source-to-image)\n",
    "\n",
    "```bash\n",
    "pip install seldon-core\n",
    "```\n",
    "\n",
    "## Train locally\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-b7995d30f035>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow/python/util/tf_should_use.py:118: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "0.9109\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot = True)\n",
    "import tensorflow as tf\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    \n",
    "    x = tf.placeholder(tf.float32, [None,784], name=\"x\")\n",
    "\n",
    "    W = tf.Variable(tf.zeros([784,10]))\n",
    "    b = tf.Variable(tf.zeros([10]))\n",
    "\n",
    "    y = tf.nn.softmax(tf.matmul(x,W) + b, name=\"y\")\n",
    "\n",
    "    y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "\n",
    "    cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))\n",
    "\n",
    "    train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "    init = tf.initialize_all_variables()\n",
    "\n",
    "    sess = tf.Session()\n",
    "    sess.run(init)\n",
    "\n",
    "    for i in range(1000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "\n",
    "    correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict = {x: mnist.test.images, y_:mnist.test.labels}))\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "    saver.save(sess, \"model/deep_mnist_model\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wrap model using s2i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "---> Installing dependencies ...\n",
      "Looking in links: /whl\n",
      "Requirement already satisfied: tensorflow>=1.12.0 in /usr/local/lib/python3.7/site-packages (from -r requirements.txt (line 1)) (1.14.0)\n",
      "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.14.0)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.11.2)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.12.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.1.7)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.7.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.1.0)\n",
      "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.2.2)\n",
      "Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.14.0)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.0.8)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.33.1)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.8.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.1.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (3.9.1)\n",
      "Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.17.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.22.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow>=1.12.0->-r requirements.txt (line 1)) (3.1.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow>=1.12.0->-r requirements.txt (line 1)) (41.0.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.15.5)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.7/site-packages (from keras-applications>=1.0.6->tensorflow>=1.12.0->-r requirements.txt (line 1)) (2.9.0)\n",
      "WARNING: You are using pip version 19.1.1, however version 19.2.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build . seldonio/seldon-core-s2i-python37:0.13 deep-mnist:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "223e9360b237670a25ccfcec4a592d16bf4f6c9fc392ddcb79d22ed1c5df499f\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"mnist_predictor\" -d --rm -p 5000:5000 deep-mnist:0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send some random features that conform to the contract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\r\n",
      "SENDING NEW REQUEST:\r\n",
      "\r\n",
      "[[0.075 0.186 0.789 0.717 0.236 0.553 0.514 0.527 0.59  0.267 0.826 0.947\r\n",
      "  0.629 0.211 0.334 0.341 0.138 0.879 0.522 0.326 0.838 0.703 0.455 0.718\r\n",
      "  0.507 0.691 0.76  0.794 0.829 0.835 0.435 0.79  0.215 0.646 0.227 0.605\r\n",
      "  0.204 0.214 0.814 0.296 0.799 0.225 0.598 0.301 0.624 0.213 0.672 0.895\r\n",
      "  0.042 0.534 0.146 0.018 0.362 0.191 0.162 0.107 0.287 0.673 0.938 0.771\r\n",
      "  0.457 0.696 0.82  0.911 0.629 0.772 0.267 0.719 0.904 0.536 0.98  0.494\r\n",
      "  0.273 0.022 0.679 0.599 0.417 0.645 0.757 0.254 0.853 0.696 0.083 0.056\r\n",
      "  0.312 0.17  0.021 0.438 0.625 0.393 0.597 0.127 0.624 0.004 0.507 0.976\r\n",
      "  0.853 0.245 0.735 0.837 0.524 0.926 0.533 0.207 0.568 0.824 0.834 0.868\r\n",
      "  0.941 0.56  0.698 0.419 0.817 0.953 0.549 0.278 0.637 0.436 0.81  0.359\r\n",
      "  0.775 0.663 0.03  0.106 0.717 0.668 0.931 0.295 0.784 0.393 0.879 0.599\r\n",
      "  0.457 0.467 0.944 0.379 0.107 0.31  0.343 0.642 0.763 0.831 0.198 0.072\r\n",
      "  0.492 0.34  0.035 0.744 0.773 0.027 0.575 0.729 0.437 0.918 0.723 0.508\r\n",
      "  0.515 0.475 0.32  0.402 0.35  0.685 0.16  0.925 0.276 0.238 0.282 0.151\r\n",
      "  0.612 0.38  0.711 0.967 0.256 0.574 0.384 0.794 0.874 0.295 0.301 0.247\r\n",
      "  0.29  0.296 0.81  0.063 0.448 0.958 0.55  0.764 0.103 0.722 0.929 0.341\r\n",
      "  0.443 0.177 0.485 0.331 0.012 0.681 0.706 0.88  0.164 0.206 0.313 0.891\r\n",
      "  0.977 0.104 0.75  0.545 0.328 0.01  0.779 0.411 0.08  0.472 0.61  0.075\r\n",
      "  0.838 0.873 0.957 0.813 0.539 0.887 0.507 0.321 0.8   0.292 0.973 0.525\r\n",
      "  0.255 0.854 0.299 0.229 0.678 0.566 0.112 0.49  0.447 0.084 0.914 0.192\r\n",
      "  0.023 0.735 0.017 0.278 0.597 0.887 0.336 0.298 0.757 0.265 0.266 0.039\r\n",
      "  0.197 0.084 0.978 0.205 0.188 0.181 0.704 0.311 0.187 0.812 0.262 0.8\r\n",
      "  0.26  0.104 0.82  0.956 0.406 0.254 0.639 0.151 0.232 0.659 0.373 0.441\r\n",
      "  0.97  0.124 0.523 0.145 0.997 0.516 0.039 0.283 0.785 0.183 0.441 0.742\r\n",
      "  0.187 0.62  0.079 0.851 0.612 0.057 0.501 0.141 0.58  0.09  0.709 0.67\r\n",
      "  0.875 0.613 0.862 0.771 0.309 0.718 0.585 0.707 0.737 0.525 0.852 0.74\r\n",
      "  0.084 0.658 0.754 0.42  0.329 0.177 0.603 0.461 0.945 0.712 0.366 0.154\r\n",
      "  0.043 0.236 0.067 0.317 0.947 0.247 0.282 0.78  0.157 0.183 0.849 0.549\r\n",
      "  0.243 0.879 0.493 0.866 0.544 0.818 0.049 0.598 0.531 0.237 0.567 0.288\r\n",
      "  0.84  0.067 0.04  0.47  0.527 0.23  0.183 0.776 0.777 0.838 0.89  0.848\r\n",
      "  0.229 0.385 0.647 0.24  0.052 0.085 0.885 0.188 0.334 0.805 0.731 0.366\r\n",
      "  0.145 0.777 0.265 0.945 0.788 0.527 0.171 0.624 0.456 0.22  0.098 0.333\r\n",
      "  0.687 0.531 0.304 0.271 0.998 0.34  0.565 0.334 0.054 0.502 0.894 0.144\r\n",
      "  0.914 0.896 0.109 0.081 0.248 0.231 0.958 0.386 0.057 0.131 0.07  0.964\r\n",
      "  0.501 0.058 0.335 0.778 0.059 0.084 0.831 0.247 0.958 0.51  0.77  0.54\r\n",
      "  0.623 0.844 0.259 0.763 0.053 0.415 0.685 0.709 0.162 0.377 0.053 0.265\r\n",
      "  0.931 0.239 0.616 0.298 0.145 0.332 0.614 0.266 0.438 0.935 0.167 0.151\r\n",
      "  0.219 0.683 0.768 0.19  0.618 0.548 0.177 0.68  0.456 0.211 0.807 0.874\r\n",
      "  0.84  0.385 0.595 0.483 0.813 0.1   0.921 0.272 0.788 0.667 0.838 0.014\r\n",
      "  0.697 0.816 0.181 0.848 0.442 0.365 0.838 0.651 0.342 0.649 0.243 0.666\r\n",
      "  0.832 0.124 0.43  0.629 0.059 0.498 0.704 0.137 0.49  0.444 0.957 0.352\r\n",
      "  0.69  0.05  0.544 0.88  0.396 0.886 0.063 0.258 0.385 0.873 0.965 0.325\r\n",
      "  0.144 0.892 0.517 0.882 0.851 0.949 0.845 0.257 0.238 0.975 0.926 0.324\r\n",
      "  0.302 0.443 0.275 0.378 0.74  0.383 0.506 0.275 0.157 0.902 0.151 0.649\r\n",
      "  0.534 0.251 0.612 0.774 0.692 0.157 0.74  0.973 0.468 0.751 0.865 0.447\r\n",
      "  0.158 0.741 0.059 0.722 0.059 0.949 0.859 0.342 0.206 0.838 0.563 0.731\r\n",
      "  0.147 0.441 0.347 0.828 0.755 0.738 0.982 0.691 0.885 0.567 0.18  0.631\r\n",
      "  0.649 0.914 0.205 0.834 0.042 0.931 0.194 0.753 0.126 0.767 0.557 0.846\r\n",
      "  0.924 0.747 0.278 0.368 0.277 0.466 0.926 0.488 0.077 0.429 0.951 0.973\r\n",
      "  0.085 0.417 0.832 0.741 0.268 0.333 0.354 0.102 0.384 0.521 0.355 0.806\r\n",
      "  0.887 0.11  0.708 0.348 0.6   0.055 0.38  0.041 0.679 0.285 0.379 0.624\r\n",
      "  0.889 0.322 0.607 0.598 0.149 0.999 0.243 0.786 0.506 0.813 0.475 0.334\r\n",
      "  0.43  0.345 0.105 0.253 0.09  0.068 0.467 0.027 0.143 0.505 0.555 0.203\r\n",
      "  0.734 0.236 0.844 0.377 0.85  0.137 0.345 0.851 0.795 0.979 0.945 0.323\r\n",
      "  0.088 0.148 0.408 0.003 0.047 0.156 0.489 0.242 0.597 0.339 0.986 0.406\r\n",
      "  0.521 0.438 0.888 0.754 0.718 0.072 0.2   0.281 0.518 0.881 0.401 0.062\r\n",
      "  0.239 0.857 0.07  0.032 0.656 0.6   0.101 0.725 0.608 0.273 0.251 0.534\r\n",
      "  0.431 0.837 0.232 0.444 0.2   0.463 0.513 0.348 0.061 0.151 0.854 0.193\r\n",
      "  0.36  0.562 0.84  0.019 0.833 0.118 0.978 0.684 0.671 0.42  0.153 0.142\r\n",
      "  0.081 0.012 0.652 0.947 0.401 0.025 0.177 0.546 0.361 0.925 0.413 0.773\r\n",
      "  0.176 0.466 0.339 0.583 0.014 0.935 0.925 0.246 0.854 0.712 0.796 0.166\r\n",
      "  0.692 0.568 0.976 0.285 0.999 0.002 0.022 0.852 0.546 0.577 0.606 0.981\r\n",
      "  0.268 0.552 0.817 0.915 0.725 0.101 0.913 0.371 0.754 0.749 0.348 0.099\r\n",
      "  0.127 0.984 0.686 0.103 0.308 0.435 0.411 0.892 0.559 0.056 0.533 0.609\r\n",
      "  0.162 0.3   0.015 0.078 0.324 0.018 0.25  0.242 0.627 0.647 0.489 0.576\r\n",
      "  0.37  0.391 0.172 0.268]]\r\n",
      "RECEIVED RESPONSE:\r\n",
      "meta {\r\n",
      "}\r\n",
      "data {\r\n",
      "  names: \"class:0\"\r\n",
      "  names: \"class:1\"\r\n",
      "  names: \"class:2\"\r\n",
      "  names: \"class:3\"\r\n",
      "  names: \"class:4\"\r\n",
      "  names: \"class:5\"\r\n",
      "  names: \"class:6\"\r\n",
      "  names: \"class:7\"\r\n",
      "  names: \"class:8\"\r\n",
      "  names: \"class:9\"\r\n",
      "  ndarray {\r\n",
      "    values {\r\n",
      "      list_value {\r\n",
      "        values {\r\n",
      "          number_value: 0.0017233663238584995\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 4.321051516598118e-09\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 0.33740362524986267\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 0.02863745018839836\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 2.708879662804975e-07\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 0.6254275441169739\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 0.0008789464482106268\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 2.0615912944776937e-05\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 0.00590110570192337\r\n",
      "        }\r\n",
      "        values {\r\n",
      "          number_value: 7.070970696076984e-06\r\n",
      "        }\r\n",
      "      }\r\n",
      "    }\r\n",
      "  }\r\n",
      "}\r\n",
      "\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-tester contract.json 0.0.0.0 5000 -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mnist_predictor\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm mnist_predictor --force"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test using Minikube\n",
    "\n",
    "**Due to a [minikube/s2i issue](https://github.com/SeldonIO/seldon-core/issues/253) you will need [s2i >= 1.1.13](https://github.com/openshift/source-to-image/releases/tag/v1.1.13)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "😄  minikube v0.34.1 on linux (amd64)\n",
      "🔥  Creating virtualbox VM (CPUs=2, Memory=4096MB, Disk=20000MB) ...\n",
      "📶  \"minikube\" IP address is 192.168.99.100\n",
      "🐳  Configuring Docker as the container runtime ...\n",
      "✨  Preparing Kubernetes environment ...\n",
      "🚜  Pulling images required by Kubernetes v1.13.3 ...\n",
      "🚀  Launching Kubernetes v1.13.3 using kubeadm ... \n",
      "🔑  Configuring cluster permissions ...\n",
      "🤔  Verifying component health .....\n",
      "💗  kubectl is now configured to use \"minikube\"\n",
      "🏄  Done! Thank you for using minikube!\n"
     ]
    }
   ],
   "source": [
    "!minikube start --memory 4096"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clusterrolebinding.rbac.authorization.k8s.io/kube-system-cluster-admin created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!helm init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"tiller-deploy\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"tiller-deploy\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Wed Aug  7 16:44:57 2019\n",
      "NAMESPACE: seldon-system\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ClusterRole\n",
      "NAME                          AGE\n",
      "seldon-operator-manager-role  1s\n",
      "\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME                                 AGE\n",
      "seldon-operator-manager-rolebinding  1s\n",
      "\n",
      "==> v1/ConfigMap\n",
      "NAME                     DATA  AGE\n",
      "seldon-config            1     1s\n",
      "seldon-spartakus-config  1     1s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                        READY  STATUS             RESTARTS  AGE\n",
      "seldon-operator-controller-manager-0        0/1    ContainerCreating  0         1s\n",
      "seldon-spartakus-volunteer-df68b9d48-hzzbd  0/1    ContainerCreating  0         1s\n",
      "\n",
      "==> v1/Secret\n",
      "NAME                                   TYPE    DATA  AGE\n",
      "seldon-operator-webhook-server-secret  Opaque  0     1s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                                        TYPE       CLUSTER-IP      EXTERNAL-IP  PORT(S)  AGE\n",
      "seldon-operator-controller-manager-service  ClusterIP  10.110.184.41   <none>       443/TCP  1s\n",
      "webhook-server-service                      ClusterIP  10.109.225.134  <none>       443/TCP  1s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME                              SECRETS  AGE\n",
      "seldon-core-seldon-core-operator  1        1s\n",
      "seldon-spartakus-volunteer        1        1s\n",
      "\n",
      "==> v1/StatefulSet\n",
      "NAME                                READY  AGE\n",
      "seldon-operator-controller-manager  0/1    1s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME                        AGE\n",
      "seldon-spartakus-volunteer  1s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME                        AGE\n",
      "seldon-spartakus-volunteer  1s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                         AGE\n",
      "seldondeployments.machinelearning.seldon.io  1s\n",
      "\n",
      "==> v1beta1/Deployment\n",
      "NAME                        READY  UP-TO-DATE  AVAILABLE  AGE\n",
      "seldon-spartakus-volunteer  0/1    1           0          1s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usageMetrics.enabled=true   --namespace seldon-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for 1 pods to be ready...\n",
      "partitioned roll out complete: 1 new pods have been updated...\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Ingress\n",
    "Please note: There are reported gRPC issues with ambassador (see https://github.com/SeldonIO/seldon-core/issues/473)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   ambassador\n",
      "LAST DEPLOYED: Wed Aug  7 16:45:10 2019\n",
      "NAMESPACE: default\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Deployment\n",
      "NAME        READY  UP-TO-DATE  AVAILABLE  AGE\n",
      "ambassador  0/3    3           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                         READY  STATUS             RESTARTS  AGE\n",
      "ambassador-6d49d6b5c7-d77lw  0/1    ContainerCreating  0         0s\n",
      "ambassador-6d49d6b5c7-m4f2h  0/1    ContainerCreating  0         0s\n",
      "ambassador-6d49d6b5c7-rcb5n  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME               TYPE          CLUSTER-IP      EXTERNAL-IP  PORT(S)                     AGE\n",
      "ambassador         LoadBalancer  10.98.88.166    <pending>    80:31622/TCP,443:30825/TCP  0s\n",
      "ambassador-admins  ClusterIP     10.109.108.136  <none>       8877/TCP                    0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "ambassador  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                          AGE\n",
      "authservices.getambassador.io                 0s\n",
      "consulresolvers.getambassador.io              0s\n",
      "kubernetesendpointresolvers.getambassador.io  0s\n",
      "kubernetesserviceresolvers.getambassador.io   0s\n",
      "mappings.getambassador.io                     0s\n",
      "modules.getambassador.io                      0s\n",
      "ratelimitservices.getambassador.io            0s\n",
      "tcpmappings.getambassador.io                  0s\n",
      "tlscontexts.getambassador.io                  0s\n",
      "tracingservices.getambassador.io              0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Congratuations! You've successfully installed Ambassador.\n",
      "\n",
      "For help, visit our Slack at https://d6e.co/slack or view the documentation online at https://www.getambassador.io.\n",
      "\n",
      "To get the IP address of Ambassador, run the following commands:\n",
      "NOTE: It may take a few minutes for the LoadBalancer IP to be available.\n",
      "     You can watch the status of by running 'kubectl get svc -w  --namespace default ambassador'\n",
      "\n",
      "  On GKE/Azure:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace default ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n",
      "\n",
      "  On AWS:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace default ambassador -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n",
      "\n",
      "  echo http://$SERVICE_IP:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install stable/ambassador --name ambassador --set crds.keep=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"ambassador\" rollout to finish: 0 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 1 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 2 of 3 updated replicas are available...\n",
      "deployment \"ambassador\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deployment.apps/ambassador"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrap Model and Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "---> Installing dependencies ...\n",
      "Looking in links: /whl\n",
      "Requirement already satisfied: tensorflow>=1.12.0 in /usr/local/lib/python3.7/site-packages (from -r requirements.txt (line 1)) (1.14.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.8.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.1.0)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.7.1)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.1.0)\n",
      "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.2.2)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.11.2)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.0.8)\n",
      "Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.17.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (3.9.1)\n",
      "Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.14.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.33.4)\n",
      "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.14.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.12.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.1.7)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 1)) (1.22.0)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.7/site-packages (from keras-applications>=1.0.6->tensorflow>=1.12.0->-r requirements.txt (line 1)) (2.9.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/site-packages (from protobuf>=3.6.1->tensorflow>=1.12.0->-r requirements.txt (line 1)) (41.0.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow>=1.12.0->-r requirements.txt (line 1)) (3.1.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow>=1.12.0->-r requirements.txt (line 1)) (0.15.5)\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build . seldonio/seldon-core-s2i-python37:0.13 deep-mnist:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/deep-mnist created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f deep_mnist.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"deep-mnist-single-model-8969cc0\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"deep-mnist-single-model-8969cc0\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/deep-mnist-single-model-8969cc0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[0.891 0.499 0.792 0.386 0.739 0.092 0.986 0.789 0.758 0.109 0.267 0.834\n",
      "  0.542 0.025 0.02  0.16  0.6   0.324 0.71  0.515 0.346 0.943 0.109 0.455\n",
      "  0.243 0.023 0.901 0.465 0.249 0.442 0.8   0.875 0.772 0.588 0.995 0.578\n",
      "  0.254 0.461 0.885 0.068 0.157 0.489 0.486 0.951 0.123 0.052 0.72  0.515\n",
      "  0.002 0.122 0.035 0.04  0.368 0.373 0.447 0.452 0.344 0.323 0.673 0.145\n",
      "  0.206 0.721 0.749 0.759 0.184 0.86  0.061 0.304 0.429 0.621 0.723 0.916\n",
      "  0.334 0.452 0.883 0.391 0.861 0.686 0.846 0.316 0.987 0.853 0.231 0.06\n",
      "  0.763 0.215 0.119 0.001 0.234 0.717 0.765 0.42  0.71  0.605 0.166 0.192\n",
      "  0.726 0.133 0.785 0.307 0.7   0.187 0.153 0.704 0.1   0.255 0.155 0.555\n",
      "  0.89  0.312 0.884 0.85  0.361 0.992 0.061 0.781 0.678 0.537 0.292 0.007\n",
      "  0.951 0.46  0.585 0.338 0.552 0.751 0.842 0.31  0.343 0.149 0.712 0.011\n",
      "  0.209 0.412 0.859 0.859 0.159 0.366 0.644 0.196 0.347 0.527 0.823 0.737\n",
      "  0.341 0.258 0.605 0.441 0.982 0.765 0.037 0.278 0.116 0.64  0.097 0.866\n",
      "  0.518 0.683 0.843 0.16  0.089 0.146 0.602 0.933 0.521 0.377 0.579 0.579\n",
      "  0.831 0.44  0.688 0.146 0.897 0.579 0.831 0.772 0.209 0.965 0.91  0.498\n",
      "  0.196 0.375 0.99  0.34  0.626 0.648 0.291 0.15  0.417 0.732 0.175 0.934\n",
      "  0.324 0.307 0.086 0.996 0.628 0.212 0.085 0.084 0.161 0.265 0.869 0.047\n",
      "  0.529 0.682 0.422 0.941 0.044 0.487 0.315 0.733 0.472 0.996 0.872 0.653\n",
      "  0.913 0.176 0.602 0.509 0.554 0.543 0.104 0.965 0.065 0.834 0.843 0.076\n",
      "  0.02  0.537 0.676 0.365 0.962 0.64  0.583 0.938 0.316 0.862 0.686 0.098\n",
      "  0.348 0.144 0.91  0.469 0.289 0.836 0.15  0.062 0.77  0.267 0.192 0.051\n",
      "  0.101 0.204 0.823 0.706 0.187 0.781 0.118 0.195 0.957 0.001 0.506 0.458\n",
      "  0.011 0.683 0.608 0.701 0.65  0.837 0.146 0.372 0.174 0.662 0.505 0.234\n",
      "  0.973 0.424 0.769 0.449 0.487 0.1   0.67  0.537 0.382 0.601 0.351 0.242\n",
      "  0.254 0.433 0.475 0.385 0.406 0.134 0.69  0.538 0.299 0.173 0.662 0.199\n",
      "  0.544 0.937 0.855 0.745 0.313 0.877 0.485 0.595 0.685 0.022 0.453 0.394\n",
      "  0.893 0.31  0.36  0.007 0.957 0.955 0.124 0.642 0.408 0.506 0.44  0.069\n",
      "  0.33  0.701 0.813 0.374 0.575 0.862 0.923 0.014 0.091 0.765 0.247 0.996\n",
      "  0.42  0.919 0.797 0.414 0.598 0.372 0.755 0.569 0.065 0.993 0.139 0.623\n",
      "  0.411 0.575 0.244 0.569 0.633 0.116 0.383 0.612 0.948 0.969 0.598 0.243\n",
      "  0.789 0.78  0.534 0.083 0.292 0.917 0.045 0.898 0.548 0.913 0.038 0.671\n",
      "  0.669 0.103 0.482 0.322 0.542 0.012 0.662 0.561 0.763 0.183 0.92  0.926\n",
      "  0.213 0.081 0.259 0.314 0.06  0.419 0.479 0.419 0.72  0.981 0.202 0.124\n",
      "  0.407 0.116 0.015 0.001 0.601 0.837 0.398 0.221 0.267 0.19  0.781 0.796\n",
      "  0.466 0.736 0.598 0.841 0.5   0.544 0.701 0.586 0.051 0.519 0.872 0.27\n",
      "  0.126 0.771 0.676 0.202 0.375 0.763 0.775 0.723 0.098 0.952 0.236 0.681\n",
      "  0.853 0.965 0.479 0.92  0.028 0.113 0.461 0.148 0.076 0.968 0.166 0.762\n",
      "  0.863 0.902 0.97  0.413 0.739 0.203 0.988 0.619 0.133 0.784 0.562 0.039\n",
      "  0.24  0.586 0.888 0.769 0.329 0.384 0.145 0.408 0.316 0.766 0.947 0.13\n",
      "  0.258 0.45  0.462 0.484 0.624 0.963 0.997 0.476 0.592 0.723 0.861 0.307\n",
      "  0.342 0.236 0.895 0.635 0.133 0.199 0.434 0.636 0.034 0.872 0.819 0.248\n",
      "  0.06  0.86  0.806 0.081 0.3   0.763 0.04  0.318 0.826 0.778 0.31  0.139\n",
      "  0.273 0.883 0.163 0.11  0.119 0.628 0.741 0.499 0.052 0.662 0.808 0.739\n",
      "  0.092 0.848 0.373 0.516 0.354 0.518 0.745 0.403 0.28  0.564 0.425 0.533\n",
      "  0.44  0.983 0.301 0.771 0.383 0.377 0.737 0.11  0.39  0.122 0.541 0.32\n",
      "  0.528 0.615 0.533 0.872 0.944 0.206 0.009 0.279 0.527 0.418 0.382 0.729\n",
      "  0.949 0.463 0.561 0.869 0.832 0.688 0.275 0.209 0.47  0.195 0.434 0.883\n",
      "  0.277 0.831 0.331 0.577 0.735 0.615 0.406 0.267 0.226 0.102 0.303 0.186\n",
      "  0.543 0.478 0.483 0.943 0.698 0.036 0.05  0.291 0.587 0.351 0.172 0.628\n",
      "  0.088 0.65  0.605 0.458 0.583 0.227 0.443 0.535 0.355 0.825 0.62  0.6\n",
      "  0.018 0.539 0.809 0.287 0.812 0.769 0.475 0.813 0.333 0.075 0.458 0.32\n",
      "  0.627 0.021 0.594 0.121 0.759 0.566 0.539 0.995 0.157 0.275 0.142 0.786\n",
      "  0.264 0.272 0.338 0.984 0.668 0.369 0.635 0.963 0.718 0.062 0.961 0.697\n",
      "  0.669 0.116 0.955 0.924 0.281 0.964 0.321 0.838 0.705 0.02  0.215 0.542\n",
      "  0.918 0.234 0.122 0.606 0.49  0.313 0.76  0.29  0.905 0.914 0.883 0.513\n",
      "  0.242 0.174 0.062 0.132 0.703 0.11  0.899 0.895 0.853 0.91  0.426 0.448\n",
      "  0.918 0.034 0.014 0.359 0.601 0.809 0.644 0.083 0.319 0.827 0.824 0.153\n",
      "  0.061 0.742 0.169 0.79  0.435 0.682 0.108 0.604 0.69  0.757 0.44  0.408\n",
      "  0.24  0.848 0.679 0.178 0.128 0.98  0.08  0.496 0.584 0.145 0.837 0.22\n",
      "  0.97  0.82  0.509 0.218 0.209 0.564 0.429 0.101 0.609 0.754 0.469 0.18\n",
      "  0.607 0.243 0.316 0.548 0.845 0.959 0.319 0.761 0.038 0.057 0.921 0.176\n",
      "  0.456 0.394 0.179 0.309 0.272 0.358 0.868 0.044 0.263 0.049 0.613 0.692\n",
      "  0.497 0.518 0.976 0.455 0.041 0.212 0.774 0.75  0.922 0.62  0.272 0.064\n",
      "  0.917 0.167 0.793 0.624 0.051 0.897 0.373 0.485 0.562 0.165 0.881 0.35\n",
      "  0.758 0.06  0.295 0.157 0.379 0.106 0.123 0.224 0.085 0.407 0.974 0.747\n",
      "  0.845 0.223 0.042 0.31 ]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "  puid: \"iq1c3r0pocs8ortlf7ebpq7tgl\"\n",
      "  requestPath {\n",
      "    key: \"classifier\"\n",
      "    value: \"deep-mnist:0.1\"\n",
      "  }\n",
      "}\n",
      "data {\n",
      "  names: \"class:0\"\n",
      "  names: \"class:1\"\n",
      "  names: \"class:2\"\n",
      "  names: \"class:3\"\n",
      "  names: \"class:4\"\n",
      "  names: \"class:5\"\n",
      "  names: \"class:6\"\n",
      "  names: \"class:7\"\n",
      "  names: \"class:8\"\n",
      "  names: \"class:9\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 0.0013847836526110768\n",
      "        }\n",
      "        values {\n",
      "          number_value: 5.2278528173133054e-09\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.45779240131378174\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.10970422625541687\n",
      "        }\n",
      "        values {\n",
      "          number_value: 6.305293709374382e-08\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.4219036400318146\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0001055227912729606\n",
      "        }\n",
      "        values {\n",
      "          number_value: 8.234503911808133e-05\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.00895524863153696\n",
      "        }\n",
      "        values {\n",
      "          number_value: 7.178235682658851e-05\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-api-tester contract.json `minikube ip` `kubectl get svc ambassador -o jsonpath='{.spec.ports[0].nodePort}'` \\\n",
    "    deep-mnist --namespace default -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🔥  Deleting \"minikube\" from virtualbox ...\n",
      "💔  The \"minikube\" cluster has been deleted.\n"
     ]
    }
   ],
   "source": [
    "!minikube delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
